package com.googlecode.gaal.preprocess.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.googlecode.gaal.preprocess.api.Normalizer;
import com.googlecode.gaal.preprocess.api.Tokenizer;

public class RegexTokenizer implements Tokenizer<String> {
    private final BufferedReader reader;
    private final Normalizer<String> normalizer;
    private final Matcher matcher;

    public RegexTokenizer(final Reader reader, final String pattern, final Normalizer<String> normalizer) {
        this(reader, Pattern.compile(pattern, Pattern.COMMENTS), normalizer);
    }

    public RegexTokenizer(final Reader reader, final Pattern pattern, final Normalizer<String> normalizer) {
        this.reader = new BufferedReader(reader);
        this.matcher = pattern.matcher("");
        this.normalizer = normalizer;
    }

    private String nextToken() {
        while (!matcher.find()) {
            String line = null;
            try {
                try {
                    if ((line = reader.readLine()) != null) {
                        matcher.reset(line);
                    } else {
                        return null;
                    }
                } finally {
                    if (line == null) {
                        reader.close();
                    }
                }
            } catch (IOException e) {
                throw new IllegalStateException(e);
            }
        }
        String token = null;
        while ((token = normalizer.normalize(matcher.group())) == null) {
            if (!matcher.find())
                return null;
        }
        return token;
    }

    @Override
    public Iterator<Document<String>> iterator() {
        return new Iterator<Document<String>>() {
            private boolean isFirst = true;

            @Override
            public boolean hasNext() throws IllegalStateException {
                return isFirst;
            }

            @Override
            public Document<String> next() {
                isFirst = false;
                return new Document<String>() {
                    @Override
                    public Iterator<String> iterator() {
                        return new Iterator<String>() {
                            String next = nextToken();

                            @Override
                            public boolean hasNext() throws IllegalStateException {
                                return next != null;
                            }

                            @Override
                            public String next() {
                                String token = next;
                                next = nextToken();
                                return token;
                            }

                            @Override
                            public void remove() {
                                throw new UnsupportedOperationException();
                            }
                        };
                    }

                    @Override
                    public int getId() {
                        return 0;
                    }
                };
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    }
}
